#! /usr/bin/python
#####################################################################################################
# Target   : Autoconvert J750 tester ASCII datalog file to customer data                            #
# Author   : Roger Zheng                                                                            #
# Version  : 00                                                                                     #
# Date	   : 01/Jul/06                                                                              #
# Location : BeiJing,China                                                                          #
#---------------------------------------------------------------------------------------------------#
#Instruction:                                                                                       #
#---------------------------------------------------------------------------------------------------#
#Filename Format: Devicename_Version_Sitnum_TestType_LotWaferId_Date_Time_Testername.data           #
#Configure File Format:                                                                             #
# Device:Version:Patch:Sortnum:Total_test_itemnum:Parameter_fuction_itemnum:bindefine:testitem_name #
#Directory Sturcture:                                                                               #
#  /TestData                                                                                        #
#       |---/Device1                                                                                #
#       |       |-----/Lot1                                                                         #
#       |       |       |---/Data   # Put original test data in this directory                      # 
#       |       |       |     |------data file1                                                     #
#       |       |       |     |------data file2                                                     #
#       |       |       |     .................                                                     #
#       |       |       |---/Retest #Put Retest data in this directory                              #
#       |       |       |     |------retest data file1                                              #
#       |       |       |     .................                                                     #
#       |       |       |---/bhtc                                                                   #
#       |       |       |---/map                                                                    #
#       |       |       |---/maphh                                                                  #
#       |       |-----/Lot2                                                                         #
#       |       ...............................                                                     #
#       |---/Device2                                                                                #
#       .......................................                                                     #
#                                                                                                   #
#Initialize Direcotry & File :                                                                      #
#$dir_testdata,$dir_confige,$file_passlotlog,$file_passwaferlog,$file_errorlog                      #
#Output:BHTC,MAP,MAPHH                                                                              #
#***************************************************************************************************#
#The script can autocovert the all target directory data to customer data. and skip the wafer or    #
#lot and that have coverted. If you want to reconvert some wafer data, you must delete the wafer    #
#information from passwafer log file and passlot log file according to data file!                   #
#####################################################################################################
####################################################################
#Note:Sort0 File in Retest directory is as Sort2 retest file
#
import os
import sys
import time
import glob
import re
import wx
import string
import shutil
import zipfile

from Configure_ver00 import Config
from Data2Bhtc_ver01 import Data2BHTC
from BHTC2CustData_ver00  import BHTC2CustData
from renamefile import Rename


def main():

  #######################################
  #Edit Area
  ######################################
  ConfigureFile="Config_OldUi"
  
  #CurrentPath=os.getcwd()
  CurrentDir=os.path.dirname(CurrentPath)
  TestDataPath=os.path.join(CurrentDir,"NewTestData")
  ConfigPath=os.path.join(CurrentPath,"Config")
  LogPath=os.path.join(CurrentPath,"Log")
  BinDefinePath=os.path.join(CurrentPath,"Parameter")
  TempPath=os.path.join(CurrentPath,"Tmp")
  
  ###########################
  #Find InitSetting_Auto for TestDataDir
  ###########################
  InitSettingFile=os.path.join(ConfigPath,"InitSetting_Auto")
  os.chdir(ConfigPath)
  if os.path.isfile(InitSettingFile):
    InitSettingFileFH=open(InitSettingFile)
    for Line in InitSettingFileFH.readlines():
      if re.search(r"TestDataDir=",Line,re.I):
         Line=Line.strip()  
         TempArray=re.split(r"=",Line)
         TestDataPath=TempArray[1]
         break
  
  if not os.path.isdir(TempPath):
    os.mkdir(TempPath)
    
  
  if not os.path.isdir(os.path.join(TempPath,"orgbhtc")):
    os.mkdir(os.path.join(TempPath,"orgbhtc"))
    
  ##############################################
  #Initial file directory!
  #############################################
  
  TempFileBHTC=os.path.join(TempPath,"bhtc_auto")
  TempOrgFileBHTC=os.path.join(TempPath,"orgbhtc")
  ErrorLogFile=os.path.join(LogPath,"error.log")
  
  try:
    ErrorFH=open(ErrorLogFile,"a")
    
  except IOError:
    print "Error Log File can't find in ",LogPath
    #ErrorFH.close()
    sys.exit()
  
  TimeConvert=time.asctime()
  
  try:
    os.chdir(TestDataPath)
    
  except OSError:
    ErrorFH=open(ErrorLogFile,"a")
    ErrorFH.write("%20s  %s%s%s" % (str(TimeConvert),"The TestDataPath:\"",TestDataPath,"\"isn't exist!\n")),
    ErrorFH.close()
    sys.exit()
  
    
  #############################################################
  #covert all data to customer data of every directory
  ############################################################  
  ExisitDevices=glob.glob("*")
  
  for DeviceName in ExisitDevices:
    DevicePath=os.path.join(TestDataPath,DeviceName)
    if os.path.isdir(DevicePath):    
      os.chdir(DevicePath)    
      ExisitLots=glob.glob("*")
      
      
      FlagPassLot=False 
      #PassLotLogFH.write("hhh\n")
      
      
      for Lot in ExisitLots: 
        ##################################################
        #Juadge if the lot have coverted
        ##################################################
        try:
          PassLotLogFH=open(os.path.join(LogPath,"passlot.log"),"r")
        except IOError:
          ErrorFH=open(ErrorLogFile)
          ErrorFH.write("%s%s%s"% ("Can't find logfile: passlot.log in directory:",LogPath,"\n"))
          ErrorFH.close()
        for LotLogLine in PassLotLogFH.readlines():
          if re.search(Lot,LotLogLine,re.I) and re.search(DeviceName,LotLogLine,re.I):
            FlagPassLot=True
            break
        PassLotLogFH.close      
        ##################################################
        #Start new lot that don't converted!
        ################################################## 
        LotPath=os.path.join(DevicePath,Lot)
        if (not FlagPassLot) and os.path.isdir(LotPath): 
          
          DataPath=os.path.join(LotPath,"data") 
          RetestDataPath=os.path.join(LotPath,"Retest")
          os.chdir(DataPath)
          ###############################################
          #For Old UI because the file format is txt
          ###############################################
          if len(glob.glob("*.txt")):
            Rename(DataPath,"txt","data")
          
          ExisitWaferFiles=sorted(glob.glob("*.data"))
          
          for WaferFile in ExisitWaferFiles:
            TempDataFileName=re.split(r"_",WaferFile)
            DeviceName=TempDataFileName[0]          
            SortType=TempDataFileName[3]
            LotWaferId=TempDataFileName[4]
            Version=""
            Patch=""
            TesterName=""
            OperatorName=""
            ChannelMap=""
            Title=""
            StarTime=""
            TestItemNum=""
            DigitalTestItemNum=""
            TestItemName=""
            FileBHTC=""
            RetestDataFiles=[]
            FlagRetest=False
            FlagRetestDataFileExist=False
            FlagExisitBHTC=False
            FlagTest="data"
            FlagConfig=False
            #FlagData=False
            FlagRun=0
            
            FlagPassWafer=False
            
            TimeConvert=time.asctime()
            #print TimeConvert
            ###################################################
            #Juadge if the wafer have coverted
            ###################################################
            TempLotWaferId=re.split("-",LotWaferId)
            LotNum=TempLotWaferId[0]
            WaferId=int(TempLotWaferId[1])
            if len(TempLotWaferId[1])<2:
              LotWaferId="".join((TempLotWaferId[0],"-","0",TempLotWaferId[1]))
            
            try:
              PassWaferLogFH=open(os.path.join(LogPath,"passwafer.log"),"r")
            except IOError:
              print "Cant't find open logfile: passwafer.log in directory:",LogPath,"\"\n"
            #PassWaferLogFH=open(u"e:\\GUI\\dataprocess\\PythonScript\\log\\passwafer.log","rw")  
            #PassWaferLogLines=PassWaferLogFH.readlines()
            for PassWaferLogLine in PassWaferLogFH.readlines():
              if re.search("^#",PassWaferLogLine,re.I) or len(re.findall("\|",PassWaferLogLine))<3:
                continue            
              TempArrayDatas=re.split("\|",PassWaferLogLine)
              Temp=re.split("-",TempArrayDatas[3])
              
              if (LotNum == Temp[0].strip())and(WaferId==int(Temp[1])) and (SortType == TempArrayDatas[4].strip())and (DeviceName == TempArrayDatas[1].strip()):
                FlagPassWafer=True
                break
            PassWaferLogFH.close  
            ##################################################
            #Start new wafer that don't converted!
            ################################################## 
            if not FlagPassWafer:  
              ######################################################
              #Got all configure parameter from configure file
              ######################################################
              (FlagConfig,Version,Patch,TestItemNum,DigitalTestItemNum,TestItemNames)=Config(DeviceName,SortType,ConfigPath,ConfigureFile)
              
              if FlagConfig:                          
                ##################################
                #Delete Temp bhtc File if exists
                ##################################
                if os.path.isfile(TempFileBHTC):
                  os.unlink(TempFileBHTC)
                if os.path.isdir(RetestDataPath):
                  os.chdir(RetestDataPath)
                  ###############################################
                  #For Old UI because the file format is txt
                  ###############################################
                  if len(glob.glob("*.txt")):
                    Rename(RetestDataPath,"txt","data")
                  TempRetestDataFiles=glob.glob("*.data")
                  TempFiles=glob.glob("*.data")
                  Date=[]
                  Time=[]
                  RetestDataFileNum=0
                  ###################################################
                  #Find the all retest file of the related wafer
                  #and sort the retest files by tested time
                  ###################################################
                  
                  for TempRetestDataFile in TempFiles:
                    TempArrayDatas=re.split("_",TempRetestDataFile)
                    Temp=re.split("-",TempArrayDatas[4])
                    ReSortType="Sort0"                  
                    if (re.search(LotNum,TempRetestDataFile,re.I))and (WaferId ==int(Temp[1]))and (string.upper(SortType) == string.upper(TempArrayDatas[3]) or string.upper(ReSortType)==string.upper(TempArrayDatas[3])) :
                      TempDatas=re.split(r"_|\.",TempRetestDataFile)
                      Date.append(TempDatas[5])
                      Time.append(TempDatas[6])
                      RetestDataFileNum=RetestDataFileNum+1
                      FlagRetest=True
                    else:
                      TempRetestDataFiles.remove(TempRetestDataFile)
                  TempDate=""
                  TempTime=""
                  
                  for Index1 in range(RetestDataFileNum):
                    for Index2 in range(RetestDataFileNum):                            
                      if int(Date[Index1])<int(Date[Index2]):
                        TempDate=Date[Index2]
                        TempTime=Time[Index2]
                        Date[Index2]=Date[Index1]
                        Time[Index2]=Time[Index1]
                        Date[Index1]=TempDate
                        Time[Index1]=TempTime
                      elif int(Date[Index1])<int(Date[Index2]):  
                        if int(Time[Index1])<int(Time[Index2]):
                          TempDate=Date[Index2]
                          TempTime=Time[Index2]
                          Date[Index2]=Date[Index1]
                          Time[Index2]=Time[Index1]
                          Date[Index1]=TempDate
                          Time[Index1]=TempTime
                  #RetestDataFiles=[] 
                  for Index in range(RetestDataFileNum):                
                    for TempFile in TempRetestDataFiles:                  
                      if re.search(Date[Index],TempFile,re.I) and re.search(Time[Index],TempFile,re.I):
                        RetestDataFiles.append(TempFile)
                        TempRetestDataFiles.remove(TempFile)
                  
                ######################################################
                #Find if there is BHTC file when retest data exist
                ######################################################
                BHTCPath=os.path.join(LotPath,"bhtc")
                  
                if os.path.isdir(BHTCPath):
                  os.chdir(BHTCPath)
                  TempBHTCFiles=glob.glob("*.bhtc")
                  for TempFile in TempBHTCFiles:
                    TempArrayDatas=re.split("_|\.",TempFile)                  
                    if re.search(LotNum,TempFile,re.I)and (WaferId==int(TempArrayDatas[-2])):
                       shutil.copyfile(TempFile,TempFileBHTC)                      
                       FileBHTC=os.path.join(BHTCPath,TempFile)
                       FlagExisitBHTC=True
                       break
                #########################################
                #Covert data
                #########################################
                if FlagExisitBHTC and FlagRetest:
                  shutil.move(FileBHTC,TempOrgFileBHTC)
                else:
                  if os.path.isfile(FileBHTC):
                    os.unlink(FileBHTC)
                  (FlagRun,StarTime,ProgramName,TesterName,Operator,ChannelMap)=Data2BHTC(LotPath,WaferFile,TempFileBHTC,TestItemNum,TestItemNames,DigitalTestItemNum,FlagTest)  
                  ###############################################
                  #Zip the converted data file 
                  ###############################################
                  if not FlagRun:
                    os.chdir(os.path.join(LotPath,FlagTest))
                    if re.search("Sort1",SortType,re.I):
                      ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_",SortType,"_",FlagTest,".zip"))
                      if not os.path.isfile(ZipFileName):
                        FileName=zipfile.ZipFile(ZipFileName,"w")
                        FileName.write(WaferFile,None,zipfile.ZIP_DEFLATED)
                        FileName.close()
                        os.unlink(WaferFile)
                      else:
                        FlagWaferFileExist=False
                        FileName=zipfile.ZipFile(ZipFileName,"r")
                        for TempFileName in FileName.namelist():
                          if re.search(WaferFile,TempFileName,re.I):
                            FlagWaferFileExist=True
                            break
                        FileName.close()
                        if not FlagWaferFileExist:    
                          FileName=zipfile.ZipFile(ZipFileName,"a")
                          FileName.write(WaferFile,None,zipfile.ZIP_DEFLATED)
                          FileName.close()
                        os.unlink(WaferFile)
                    if re.search("Sort2",SortType,re.I) or re.search("Sort0",SortType,re.I):
                      ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_","Sort2","_",FlagTest,".zip"))
                      if not os.path.isfile(ZipFileName):
                        FileName=zipfile.ZipFile(ZipFileName,"w")
                        FileName.write(WaferFile,None,zipfile.ZIP_DEFLATED)
                        FileName.close()
                        os.unlink(WaferFile)
                      else:
                        FlagWaferFileExist=False
                        FileName=zipfile.ZipFile(ZipFileName,"r")
                        for TempFileName in FileName.namelist():
                          if re.search(WaferFile,TempFileName,re.I):
                            FlagWaferFileExist=True
                            break
                        FileName.close()
                        if not FlagWaferFileExist:    
                          FileName=zipfile.ZipFile(ZipFileName,"a")
                          FileName.write(WaferFile,None,zipfile.ZIP_DEFLATED)
                          FileName.close()
                        os.unlink(WaferFile)
                      
                if FlagRetest:
                  TempFlagRun=0
                  TempFlag=0
                  FlagTest="Retest"
                  for RetestDataFile in RetestDataFiles:
                    TempDataFileName=re.split(r"_",RetestDataFile)
                    DeviceName=TempDataFileName[0]          
                    SortType=TempDataFileName[3]
                    #LotWaferId=TempDataFileName[4]
                    (FlagConfig,Version,Patch,TestItemNum,DigitalTestItemNum,TestItemNames)=Config(DeviceName,SortType,ConfigPath,ConfigureFile)                  
                    if string.upper(SortType)==string.upper("Sort0"):
                      SortType="Sort2"
                    (TempFlag,ProgramName,TesterName,Operator,ChannelMap)=Data2BHTC(LotPath,RetestDataFile,TempFileBHTC,TestItemNum,TestItemNames,DigitalTestItemNum,FlagTest)  
                    ###############################################
                    #Zip the converted retest data file 
                    ###############################################
                    os.chdir(os.path.join(LotPath,FlagTest))
                    if re.search("Sort1",SortType,re.I):
                      ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_",SortType,"_",FlagTest,".zip"))
                      if not os.path.isfile(ZipFileName):
                        FileName=zipfile.ZipFile(ZipFileName,"w")
                        FileName.write(RetestDataFile,None,zipfile.ZIP_DEFLATED)
                        FileName.close()
                        os.unlink(RetestDataFile)
                      else:
                        FlagRetestDataFileExist=False
                        FileName=zipfile.ZipFile(ZipFileName,"r")
                        for TempFileName in FileName.namelist():
                          if re.search(RetestDataFile,TempFileName,re.I):
                            FlagRetestDataFileExist=True
                            break
                        FileName.close()
                        if not FlagRetestDataFileExist:    
                          FileName=zipfile.ZipFile(ZipFileName,"a")
                          FileName.write(RetestDataFile,None,zipfile.ZIP_DEFLATED)
                          FileName.close()
                        os.unlink(RetestDataFile)
                    if re.search("Sort2",SortType,re.I) or re.search("Sort0",SortType,re.I):
                      ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_","Sort2","_",FlagTest,".zip"))
                      if not os.path.isfile(ZipFileName):
                        FileName=zipfile.ZipFile(ZipFileName,"w")
                        FileName.write(RetestDataFile,None,zipfile.ZIP_DEFLATED)
                        FileName.close()
                        os.unlink(RetestDataFile)
                      else:
                        FlagRetestDataFileExist=False
                        FileName=zipfile.ZipFile(ZipFileName,"r")
                        for TempFileName in FileName.namelist():
                          if re.search(RetestDataFile,TempFileName,re.I):
                            FlagRetestDataFileExist=True
                            break
                        FileName.close()
                        if not FlagRetestDataFileExist:    
                          FileName=zipfile.ZipFile(ZipFileName,"a")
                          FileName.write(RetestDataFile,None,zipfile.ZIP_DEFLATED)
                          FileName.close()
                        os.unlink(RetestDataFile)
                    
                    TempFlagRun=TempFlagRun or TempFlag
                      
                  if TempFlagRun:
                    FlagRun=TempFlagRun
                ###########################################
                #Print out the related customer data
                ###########################################
                if not FlagRun:
                  FlagRun=BHTC2CustData(LotPath,DeviceName,TestItemNum,TesterName,ProgramName,Version,LotWaferId,Patch,SortType,Operator,ChannelMap,StarTime,BinDefinePath,TempFileBHTC,TestItemNames)
                  PassWaferLogFH=open(os.path.join(LogPath,"passwafer.log"),"a")
                  PassWaferLogFH.write("%26s%5s%15s%5s%20s%5s%18s%5s%12s%5s" %(TimeConvert,"|",DeviceName,"|",Lot,"|",LotWaferId,"|",SortType,"\n"))
                  PassWaferLogFH.close()
              else:
                ErrorFH=open(ErrorLogFile,"a")
                ErrorFH.write(TimeConvert),
                ErrorFH.write("\tNot found device:"),
                ErrorFH.write(DeviceName),
                ErrorFH.write("\'s configure data in file:"),
                ErrorFH.write(ConfigureFile),
                ErrorFH.write("! When Converting File:")
                ErrorFH.write(WaferFile)
                ErrorFH.write("\n")
                ErrorFH.close()
                #TempTime=time.asctime()
                #print TempTime

if __name__ == '__main__':       
    CurrentPath=os.getcwd()    
    ConfigPath=os.path.join(CurrentPath,"Config")
    ConfigFile=os.path.join(ConfigPath,"InitSetting_Auto") 
    CurrentTime=time.asctime()
    TempArray=re.split("\s+|:",CurrentTime)
    Hour=int(TempArray[3])
    Minute=int(TempArray[4])
    WaitTime=0  
    RunType=0
    RunInterval=2
    InitSettingFileFH=open(ConfigFile,'r')
    for Line in InitSettingFileFH.readlines():
      Line=Line.strip()
      if re.search("^#",Line,re.I):
    	continue
      elif re.search("RunType=",Line,re.I):
        TempArray=re.split("=",Line)
    	RunType=int(TempArray[1])
      elif re.search("RunInterval=",Line,re.I):
    	TempArray=re.split("=",Line)
    	RunInterval=float(TempArray[1])
    
    (MyInt,Residue)=divmod(Hour,2)	 		
    if RunType:
      if Residue:
        WaitTime=1*60+(60-Minute)
      else:
        WaitTime=60-Minute
    else:
      if Residue:
        WaitTime=60-Minute
      else:
        WaitTime=1*60+(60-Minute)
    
    time.sleep(WaitTime*60)
    while True:	    		        				 		
      main()
      time.sleep(RunInterval*3600)                       